{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from keras.layers import Dense, Input, Flatten\n",
    "from keras.layers import GlobalMaxPool1D, Bidirectional, Convolution1D, Embedding, BatchNormalization,MaxPooling1D, Dropout, LSTM\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints\n",
    "from keras.models import Model\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "INPUT_PATH = '../input/'\n",
    "CACHE_PATH = '../cache/'\n",
    "OUTPUT_PATH ='../output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = np.load(CACHE_PATH + 'data.npz')\n",
    "X_train = data['X_train']\n",
    "y_train = data['y_train']\n",
    "X_val = data['X_val']\n",
    "y_val = data['y_val']\n",
    "X_test = data['X_test']\n",
    "embedding_matrix = np.load(CACHE_PATH + 'embedding_matrix.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_FEATURES = 20000\n",
    "MAX_SEQUENCE_LENGTH = 300\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(MAX_FEATURES,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "adam 优化器比SGD要好，自适应学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_lstm_model():\n",
    "    inp = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(inp)\n",
    "    x = Bidirectional(LSTM(64, dropout=0.25, recurrent_dropout=0.25, return_sequences=True))(embedded_sequences)\n",
    "    x = Bidirectional(LSTM(32, dropout=0.25, recurrent_dropout=0.25, return_sequences=True))(embedded_sequences)\n",
    "    x = Bidirectional(LSTM(32, dropout=0.25, recurrent_dropout=0.25, return_sequences=True))(embedded_sequences)\n",
    "    x = GlobalMaxPool1D()(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(128, activation=\"relu\")(x)\n",
    "    x = Dropout(0.1)(x)\n",
    "    x = Dense(1, activation=\"linear\")(x)\n",
    "    model = Model(inputs=inp, outputs=x)\n",
    "    model.compile(loss='mse',optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_lstm_model(model):\n",
    "    model_path = CACHE_PATH + \"lstm_weights_best.hdf5\"\n",
    "    early = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=5)\n",
    "    checkpoint = ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')\n",
    "    callbacks_list = [checkpoint, early]\n",
    "    model.fit(X_train, y_train, batch_size=128, epochs=100, validation_data=(X_val, y_val), callbacks=callbacks_list)\n",
    "    model.load_weights(model_path)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = get_lstm_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 300, 300)          6000000   \n",
      "_________________________________________________________________\n",
      "bidirectional_3 (Bidirection (None, 300, 64)           85248     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               8320      \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 6,093,697\n",
      "Trainable params: 93,697\n",
      "Non-trainable params: 6,000,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model,to_file='./lstm.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 209000 samples, validate on 11000 samples\n",
      "Epoch 1/100\n",
      "209000/209000 [==============================] - 955s 5ms/step - loss: 0.6639 - val_loss: 0.4725\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.47251, saving model to ../cache/lstm_weights_best.hdf5\n",
      "Epoch 2/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.5078 - val_loss: 0.4485\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.47251 to 0.44850, saving model to ../cache/lstm_weights_best.hdf5\n",
      "Epoch 3/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4914 - val_loss: 0.4635\n",
      "\n",
      "Epoch 00003: val_loss did not improve\n",
      "Epoch 4/100\n",
      "209000/209000 [==============================] - 952s 5ms/step - loss: 0.4794 - val_loss: 0.4951\n",
      "\n",
      "Epoch 00004: val_loss did not improve\n",
      "Epoch 5/100\n",
      "209000/209000 [==============================] - 950s 5ms/step - loss: 0.4685 - val_loss: 0.4602\n",
      "\n",
      "Epoch 00005: val_loss did not improve\n",
      "Epoch 6/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4597 - val_loss: 0.4534\n",
      "\n",
      "Epoch 00006: val_loss did not improve\n",
      "Epoch 7/100\n",
      "209000/209000 [==============================] - 950s 5ms/step - loss: 0.4542 - val_loss: 0.4330\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.44850 to 0.43299, saving model to ../cache/lstm_weights_best.hdf5\n",
      "Epoch 8/100\n",
      "209000/209000 [==============================] - 950s 5ms/step - loss: 0.4487 - val_loss: 0.4348\n",
      "\n",
      "Epoch 00008: val_loss did not improve\n",
      "Epoch 9/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4446 - val_loss: 0.4235\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.43299 to 0.42354, saving model to ../cache/lstm_weights_best.hdf5\n",
      "Epoch 10/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4411 - val_loss: 0.4283\n",
      "\n",
      "Epoch 00010: val_loss did not improve\n",
      "Epoch 11/100\n",
      "209000/209000 [==============================] - 950s 5ms/step - loss: 0.4355 - val_loss: 0.4242\n",
      "\n",
      "Epoch 00011: val_loss did not improve\n",
      "Epoch 12/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4316 - val_loss: 0.4227\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.42354 to 0.42269, saving model to ../cache/lstm_weights_best.hdf5\n",
      "Epoch 13/100\n",
      "209000/209000 [==============================] - 950s 5ms/step - loss: 0.4290 - val_loss: 0.4202\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.42269 to 0.42024, saving model to ../cache/lstm_weights_best.hdf5\n",
      "Epoch 14/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4263 - val_loss: 0.4235\n",
      "\n",
      "Epoch 00014: val_loss did not improve\n",
      "Epoch 15/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4223 - val_loss: 0.4188\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.42024 to 0.41883, saving model to ../cache/lstm_weights_best.hdf5\n",
      "Epoch 16/100\n",
      "209000/209000 [==============================] - 950s 5ms/step - loss: 0.4200 - val_loss: 0.4162\n",
      "\n",
      "Epoch 00016: val_loss improved from 0.41883 to 0.41617, saving model to ../cache/lstm_weights_best.hdf5\n",
      "Epoch 17/100\n",
      "209000/209000 [==============================] - 950s 5ms/step - loss: 0.4192 - val_loss: 0.4181\n",
      "\n",
      "Epoch 00017: val_loss did not improve\n",
      "Epoch 18/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4168 - val_loss: 0.4175\n",
      "\n",
      "Epoch 00018: val_loss did not improve\n",
      "Epoch 19/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4154 - val_loss: 0.4163\n",
      "\n",
      "Epoch 00019: val_loss did not improve\n",
      "Epoch 20/100\n",
      "209000/209000 [==============================] - 950s 5ms/step - loss: 0.4134 - val_loss: 0.4196\n",
      "\n",
      "Epoch 00020: val_loss did not improve\n",
      "Epoch 21/100\n",
      "209000/209000 [==============================] - 951s 5ms/step - loss: 0.4114 - val_loss: 0.4374\n",
      "\n",
      "Epoch 00021: val_loss did not improve\n"
     ]
    }
   ],
   "source": [
    "model = train_lstm_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 48s 964us/step\n"
     ]
    }
   ],
   "source": [
    "y_test = model.predict(X_test,batch_size=128,verbose=1)\n",
    "y_test[y_test < 1] = 1\n",
    "y_test[y_test > 4.7] = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "time = datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv(INPUT_PATH + 'sample.csv',header=None,names=['Id','Score'])\n",
    "sub['Score'] = y_test\n",
    "sub.to_csv(OUTPUT_PATH + 'lstm_{}.csv'.format(time.strftime('%Y-%m-%d-%H:%M:%S')),index=False, header=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_predict = pd.read_csv(CACHE_PATH+'test_pred_model.csv')\n",
    "# columnsname = time.strftime(\"%d-%H-%M\")\n",
    "# df_predict[columnsname] = y_test\n",
    "# df_predict.to_csv(CACHE_PATH+'test_pred_model.csv',index=False)\n",
    "\n",
    "# df_train = pd.read_csv(CACHE_PATH + 'train_pred_model.csv')\n",
    "# df_train = df_train[df_train['Score'].notnull()].reset_index(drop=True)\n",
    "# y_train = model.predict(X_train,batch_size=128,verbose=1)\n",
    "# y_train[y_train < 1] = 1\n",
    "# y_train[y_train > 4.7] = 5\n",
    "# df_train[columnsname] = y_train\n",
    "# df_train.to_csv(CACHE_PATH+'train_pred_model.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
